package com.g2room.driving.grap.images;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.eclipse.jetty.util.StringUtil;

import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.model.po.Image;
import com.g2room.driving.model.po.Theme;
import com.g2room.driving.util.DbUtil;
import com.g2room.driving.util.ImageUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlBold;
import com.gargoylesoftware.htmlunit.html.HtmlImage;
import com.gargoylesoftware.htmlunit.html.HtmlListItem;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSpan;

public class Grab4493DataByProxy {
	static Connection connection = null;
	static PreparedStatement  ps = null;
	static Statement  statement = null;
	
	static Map<String, String> errorUrls = new HashMap<String, String>();
	private static final String dir = "E:/高绪国/资料/meinv/";
	private static final String source = "美女图片";
	
	public static void main(String[] args) throws Exception{
		
		Grab4493DataByProxy gkd = new Grab4493DataByProxy();
		gkd.getAllTags();
		
//		gkd.getPage();
//		// 最后输出失败的链接到数据库
////		gkd.getDetail("http://www.kcjq.com/news/03/2009/1214/0912141013493CDA28B3C0GB4036EDBD.html");
//		insertError2Db();
		DbUtil.closeConn();
    }
	
	public void getAllTags() {
		List<String> tags = getAllThemeTag();
		Set<String> set = new HashSet<String>();
		for(String tag : tags) {
			if(StringUtil.isBlank(tag)) continue;
			String[] a = tag.split("\\|");
			for(String aa : a) {
				set.add(aa);
			}
		}
		
		for(String tag : set) {
			System.out.println(tag);
		}
	}
	private List<String> getAllThemeTag() {
		List<String> urls = new ArrayList<String>();
		try {
            String sql = "select tags from i_theme where source='美女图片'";
            if(statemenetCount==0) statement = DbUtil.getConn().createStatement();  
            ResultSet rs = statement.executeQuery(sql);
            while(rs.next()) {
            	urls.add(rs.getString("tags"));
            }
            rs.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return urls;
	}

	public void updateTags() {
		Map<Integer, String> themes = getAllTheme();
		WebClient wc = GenericGrap.getWebClient();
		for(Integer id : themes.keySet()) {
			String url = themes.get(id);
			HtmlPage page = null;
	        try {
	        	page = wc.getPage(url);
	        } catch (Exception e) {
	        	errorUrls.put(url, "id"+id);
	        	e.printStackTrace();
	        	continue;
	        }
	        
	     // 获取Tags
	        List<?> aList = page.getByXPath("//div[@class=\"picbottomline\"]/p[@class=\"pleft\"]/span/a");
	        String tags = "";
	        for(Object o : aList) {
	        	HtmlAnchor a = (HtmlAnchor) o;
	        	tags += a.asText().trim() + "|";
	        }
	        
	        if(StringUtil.isBlank(tags)) continue;
	        // 
	        try {
		        String sql = "update i_theme set tags=? where id=?";
		        System.out.println(id + "------=" + tags);
	            if(sqlCount == 0) {
	            	ps = DbUtil.getConn().prepareStatement(sql);
	            	sqlCount++;
	            }
				ps.setString(1, tags);
				ps.setInt(2, id);
				ps.execute();
			} catch(Exception e) {
				e.printStackTrace();
			}
		}
		if (null != wc) wc.close();
	}
	private Map<Integer, String> getAllTheme() {
		Map<Integer, String> urls = new HashMap<Integer, String>();
		try {
            String sql = "select id,source_url from i_theme where source='美女图片' and tags = tags_original";
            if(statemenetCount==0) statement = DbUtil.getConn().createStatement();  
            ResultSet rs = statement.executeQuery(sql);
            while(rs.next()) {
            	urls.put(rs.getInt("id"), rs.getString("source_url"));
            }
            rs.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return urls;
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getPage() throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		int pageNum = 0;
//		String categoryName="网络美女", pageUrl ="http://www.4493.com/wangluomeinv/";
//		pageNum = 214;
		
		
		String categoryName="唯美写真", pageUrl ="http://www.4493.com/weimeixiezhen/";
		pageNum = 213;
//		String categoryName="性感美女", pageUrl ="http://www.4493.com/xingganmote/";
//		pageNum = 273;
		
//		// 已完结部分
//		String categoryName="丝袜美腿", pageUrl ="http://www.4493.com/siwameitui/";
//		pageNum = 80;
		
		
//		String categoryName="模特美女", pageUrl ="http://www.4493.com/motemeinv/";
//		pageNum = 101;
//		String categoryName="动漫美女", pageUrl ="http://www.4493.com/dongmanmeinv/";
//		pageNum = 9;
//		String categoryName="体育美女", pageUrl ="http://www.4493.com/tiyumeinv/";
//		pageNum = 34;
		
		

    	List<String> urls = getUrlTheme();
    	
		List<String> pages = new ArrayList<String>();
		for(int i = 1; i<=pageNum; i++) {
			if(i == 1) {
				pages.add(pageUrl);
			} else {
				pages.add(pageUrl + "index-"+i+".htm");
			}
		}
		for(String url : pages) {
			System.out.println("--------------start get page data!" + categoryName + ",url=" + url);
			WebClient wc = GenericGrap.getWebClient();
			//获取页面
			HtmlPage page = null;
	        try {
	        	page = wc.getPage(url);
	        } catch (Exception e) {
	        	errorUrls.put(url, categoryName);
	        	e.printStackTrace();
	        	return;
	        }
	        
			// 获取每页的图片列表
			List<?> list = page.getByXPath("//ul[@class=\"clearfix\"]/li");
			List<Theme> themes = new ArrayList<Theme>();
			for(Object hs : list) {
				Theme theme = new Theme();
				theme.setCategoryName(categoryName);
				HtmlListItem listItem = (HtmlListItem) hs;
				List<?> alist = listItem.getElementsByTagName("a");
				// 获取详细页面链接
				HtmlAnchor detailAnchor = (HtmlAnchor) alist.get(0);
				
				if(urls.contains(detailAnchor.getHrefAttribute())) {
		    		System.out.println("url 存在==" + detailAnchor.getHrefAttribute());
		    		continue;
		    	}
				String detailUrl = detailAnchor.getHrefAttribute().replace("/1.htm", ".htm");
				theme.setSourceUrl(detailUrl);
				HtmlImage img = (HtmlImage) detailAnchor.getElementsByTagName("img").get(0);
				theme.setTitle(img.getAltAttribute());
				theme.setCoverOriginal(img.getSrcAttribute());
	
				HtmlBold bold = (HtmlBold) listItem.getElementsByTagName("b").get(0);
				theme.setPublishTime(bold.asText());
				
				themes.add(theme);
			}
	        if (null != wc) wc.close();
	        
	        for(Theme theme : themes) {
	        	getDetail(theme);
	        }
		}
	}
	
	public void getDetail(Theme theme) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("*************start get detail data!  " + theme.getSourceUrl());
		WebClient wc = GenericGrap.getWebClient();
		HtmlPage page = null;
        try {
        	page = wc.getPage(theme.getSourceUrl());
        } catch (Exception e) {
        	errorUrls.put(theme.getSourceUrl(), theme.getCategoryName());
        	e.printStackTrace();
        	return;
        }
        

        // 获取Tags
        List<?> spanList = page.getByXPath("//div[@class=\"picbottomline\"]/p[@class=\"pleft\"]/span");
        if(null != spanList && !spanList.isEmpty()) {
        	HtmlSpan span = (HtmlSpan) spanList.get(0);
        	theme.setTags(span.asText().replaceAll("Tags:", "").trim());
        }
        
        String filePath = "";
		try {
			filePath = ImageUtil.downloadImage(theme.getCoverOriginal(), dir, "封面/" + theme.getCategoryName(), null);
			theme.setCover(filePath);
		} catch (Exception e) {
			errorUrls.put(theme.getCoverOriginal(), theme.getCategoryName()+"封面图片下载失败");
			e.printStackTrace();
		}
        // 插入数据库表
        theme.setId(inserTheme(theme));
        
        Image image = new Image();
		// 获取分页数据  img onload="btnaddress(1);"
        List<?> imgList = page.getByXPath("//div[@class=\"picsbox\"]/p/img");
        if(null == imgList || imgList.isEmpty()) return;
        for(Object o : imgList) {
			HtmlImage img = (HtmlImage) o;
			image.setImageOriginal(img.getSrcAttribute());
			if (null != wc) wc.close();
			image.setThemeId(theme.getId());
			try {
				filePath = ImageUtil.downloadImage(image.getImageOriginal(), dir, theme.getCategoryName(), null);
				image.setImage(filePath);
			} catch (Exception e) {
				e.printStackTrace();
			}
			// 插入数据库
			inserImage(image);
			// 错误数据每50条插入一次
			if(errorUrls.size() >50) {
				insertError2Db();
				errorUrls.clear();
			}
        }
	}


	int sqlCount = 0;
	private int inserTheme(Theme theme) {
		try {
            String sql = "insert into i_theme(title,cover,category_name,publish_time,cover_original,tags,source_url,source) values(?,?,?,?,?,?,?,?)";
            ps = DbUtil.getConn().prepareStatement(sql,Statement.RETURN_GENERATED_KEYS);  
            if(sqlCount == 0) {
            	ps = DbUtil.getConn().prepareStatement(sql);
            	sqlCount++;
            }
			ps.setString(1, theme.getTitle());
			ps.setString(2, theme.getCover());
			ps.setString(3, theme.getCategoryName());
			ps.setString(4, theme.getPublishTime());
			ps.setString(5, theme.getCoverOriginal());
			ps.setString(6, theme.getTags());
			ps.setString(7, theme.getSourceUrl());
			ps.setString(8, source);
			ps.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return getId();
	}

	private List<String> getUrlTheme() {
		List<String> urls = new ArrayList<String>();
		try {
            String sql = "select source_url from i_theme";
            if(statemenetCount==0) statement = DbUtil.getConn().createStatement();  
            ResultSet rs = statement.executeQuery(sql);
            while(rs.next()) {
            	urls.add(rs.getString("source_url"));
            }
            rs.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return urls;
	}
	int statemenetCount = 0;
	private int getId() {
		int id = 0;
		try {
            String sql = "select max(id) as id from i_theme";
            if(statemenetCount==0)  statement = DbUtil.getConn().createStatement();  
            ResultSet rs = statement.executeQuery(sql);
            while(rs.next()) {
            	id = rs.getInt("id");
            }
            rs.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return id;
	}


	int sqlImageCount = 0;
	private void inserImage(Image image) {
		try {
            String sql = "insert into i_image(theme_id,image,image_original) values(?,?,?)";
            ps = DbUtil.getConn().prepareStatement(sql,Statement.RETURN_GENERATED_KEYS);  
            if(sqlImageCount == 0) {
            	ps = DbUtil.getConn().prepareStatement(sql);
            	sqlImageCount++;
            }
			ps.setInt(1, image.getThemeId());
			ps.setString(2, image.getImage());
			ps.setString(3, image.getImageOriginal());
			ps.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void insertError2Db() {
		try {
			PreparedStatement  ps = DbUtil.getConn().prepareStatement("insert into i_grap_err(title,source_url,source) values(?,?,?)");
			System.out.println("===================error===========================");
			for(String url : errorUrls.keySet()) {
				System.out.println(url + " --------------- " + errorUrls.get(url));
				ps.setString(1, errorUrls.get(url));
				ps.setString(2, url);
				ps.setString(3, source);
				ps.addBatch();
			}
			ps.executeBatch();
			if(null != ps) ps.close();
			if(null != connection) connection.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
}
